{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>426</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_id  label\n",
       "0           0      0\n",
       "424         1      5\n",
       "426         2      5\n",
       "460         3      5\n",
       "3260        4      5"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import gc\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from tqdm import tqdm\n",
    "%matplotlib inline\n",
    "\n",
    "tqdm.pandas()\n",
    "\n",
    "from contextlib import contextmanager\n",
    "@contextmanager\n",
    "def timer(name):\n",
    "    import time\n",
    "    startTime = time.time()\n",
    "    yield\n",
    "    elapsedTime = time.time() - startTime\n",
    "    print('[{}] finished in {} s'.format(name, int(elapsedTime)))\n",
    "\n",
    "train = pd.read_csv('train.csv')\n",
    "train_data = train[['file_id','label']].drop_duplicates()\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2644. , 2524. , 2516. , ..., 2338.4, 2448. , 2668. ])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.groupby(['file_id'])['tid'].quantile(0.2).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:3: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:9: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n",
      "max\n",
      "min\n",
      "median\n",
      "std\n",
      "count\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/admin/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:20: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nunique\n",
      "max\n",
      "min\n",
      "median\n",
      "std\n"
     ]
    }
   ],
   "source": [
    "api_opt = ['count','nunique']\n",
    "for opt in api_opt:\n",
    "    tmp = train.groupby(['file_id'])['api'].agg({'fileid_api_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id')\n",
    "tid_opt = ['count','nunique','max','min','median','std'] \n",
    "\n",
    "for opt in tid_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['tid'].agg({'fileid_tid_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id')\n",
    "\n",
    "secs = [0.2,0.4,0.6,0.8]\n",
    "for sec in secs: \n",
    "    train_data['fileid_tid_quantile_' + str(sec * 100)] = train.groupby(['file_id'])['tid'].quantile(sec).values\n",
    "train_data['fileid_tid_range'] = train.groupby(['file_id'])['tid'].quantile(0.975).values - train.groupby(['file_id'])['tid'].quantile(0.0125).values\n",
    "\n",
    "index_opt = ['count','nunique','max','min','median','std'] \n",
    "for opt in index_opt:\n",
    "    print(opt)\n",
    "    tmp = train.groupby(['file_id'])['index'].agg({'fileid_index_' + opt: opt}).reset_index() \n",
    "    train_data = pd.merge(train_data,tmp,how='left', on='file_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▎    | 165/308 [33:05<28:03, 11.78s/it] Process ForkPoolWorker-5:\n",
      "Process ForkPoolWorker-7:\n",
      "Process ForkPoolWorker-10:\n",
      "Process ForkPoolWorker-11:\n",
      "Process ForkPoolWorker-2:\n",
      "Process ForkPoolWorker-9:\n",
      "Process ForkPoolWorker-4:\n",
      "Process ForkPoolWorker-8:\n",
      "Process ForkPoolWorker-12:\n",
      "Process ForkPoolWorker-1:\n",
      "Process ForkPoolWorker-6:\n",
      "Process ForkPoolWorker-3:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n",
      "    result = (True, func(*args, **kwds))\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 35, in multi_task\n",
      "    df[f'api_{name}_num'] = df['api'].apply(api_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 35, in multi_task\n",
      "    df[f'api_{name}_num'] = df['api'].apply(api_num, api_name=name)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 35, in multi_task\n",
      "    df[f'api_{name}_num'] = df['api'].apply(api_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 35, in multi_task\n",
      "    df[f'api_{name}_num'] = df['api'].apply(api_num, api_name=name)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 36, in multi_task\n",
      "    df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3192, in apply\n",
      "    mapped = lib.map_infer(values, f, convert=convert_dtype)\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"pandas/_libs/src/inference.pyx\", line 1472, in pandas._libs.lib.map_infer\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "KeyboardInterrupt\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"/home/admin/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\", line 3179, in <lambda>\n",
      "    f = lambda x: func(x, *args, **kwds)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 7, in api_rate_num\n",
      "    x = x.split(' ')\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 8, in api_rate_num\n",
      "    return x.count(api_name) / len(x)\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 3, in api_num\n",
      "    x = x.split(' ')\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 7, in api_rate_num\n",
      "    x = x.split(' ')\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 7, in api_rate_num\n",
      "    x = x.split(' ')\n",
      "KeyboardInterrupt\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 7, in api_rate_num\n",
      "    x = x.split(' ')\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 7, in api_rate_num\n",
      "    x = x.split(' ')\n",
      "  File \"<ipython-input-4-e898a534f14f>\", line 3, in api_num\n",
      "    x = x.split(' ')\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e898a534f14f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     37\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mapply_mul_core\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-4-e898a534f14f>\u001b[0m in \u001b[0;36mapply_mul_core\u001b[0;34m(df)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mresults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m     \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    544\u001b[0m         \u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'joining pool'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mCLOSE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTERMINATE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 546\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_worker_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    547\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_task_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    548\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1056\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1057\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1058\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1070\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1071\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1072\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1073\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1074\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "all_apis = set(train['api'])\n",
    "def api_num(x, api_name):\n",
    "    x = x.split(' ')\n",
    "    return x.count(api_name)\n",
    "\n",
    "def api_rate_num(x, api_name):\n",
    "    x = x.split(' ')\n",
    "    return x.count(api_name) / len(x)\n",
    "\n",
    "def apply_mul_core(df):\n",
    "    import multiprocessing as mlp\n",
    "    num_cpu = 12\n",
    "    pool = mlp.Pool(num_cpu)\n",
    "    batch_num = 1 + len(df) // num_cpu\n",
    "    results = []\n",
    "    for i in range(num_cpu):\n",
    "        task = df[i*batch_num : (i+1)*batch_num]\n",
    "        result = pool.apply_async(multi_task,(task,))\n",
    "        results.append(result)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    res = pd.DataFrame({})\n",
    "    for result in results:\n",
    "        feat = result.get()\n",
    "        res = pd.concat([res, feat])\n",
    "    return res\n",
    "\n",
    "temp = train.groupby(['file_id'])['api'].apply(lambda x:' '.join(list(x))).to_frame().reset_index()\n",
    "train_data = pd.merge(train_data,temp,how='left', on='file_id')\n",
    "\n",
    "all_apis = list(all_apis)\n",
    "\n",
    "def multi_task(df):\n",
    "    for name in tqdm(all_apis):\n",
    "        df[f'api_{name}_num'] = df['api'].apply(api_num, api_name=name)\n",
    "        df[f'api_{name}_rate_num'] = df['api'].apply(api_rate_num, api_name=name)\n",
    "    return df\n",
    "\n",
    "df = apply_mul_core(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = df.drop(['file_id','label', 'api'],axis=1).values\n",
    "train_Y = df['label'].values\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "import lightgbm as lgb\n",
    "\n",
    "skf = StratifiedKFold(train_Y, n_folds=5, shuffle=True, random_state=2018)\n",
    "for i,(tr_idx,val_idx) in enumerate(skf):\n",
    "        print('FOLD: ',i)\n",
    "        X_train,X_train_label = train_X[tr_idx],train_Y[tr_idx]\n",
    "        X_val,X_val_label = train_X[val_idx],train_Y[val_idx]\n",
    "        dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "        dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "        params = {\n",
    "                'task':'train', \n",
    "                'boosting_type':'gbdt',\n",
    "                'num_leaves': 15,\n",
    "                'objective': 'multiclass',\n",
    "                'num_class':8,\n",
    "                'learning_rate': 0.01,\n",
    "                'feature_fraction': 0.85,\n",
    "                'subsample':0.85,\n",
    "                'num_threads': 28,\n",
    "                'metric':'multi_logloss',\n",
    "                'seed':2018\n",
    "            }\n",
    "        model = lgb.train(params, \n",
    "                          dtrain, \n",
    "                          num_boost_round=100000,\n",
    "                          valid_sets=[dval],\n",
    "                          verbose_eval=100, \n",
    "                          early_stopping_rounds=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text  import TfidfVectorizer\n",
    "import os \n",
    "import pickle\n",
    "n_range=(1,4)  #\n",
    "max_feature=100000 #\n",
    "\n",
    "if os.path.exists(\"API-Tfidf_%s.pkl\"%str(n_range))==False:\n",
    "    print(\"xx\")\n",
    "    api_tfidf=TfidfVectorizer(ngram_range=n_range,max_features=max_feature,min_df=2,max_df=0.97)\n",
    "    api_tfidf.fit(train_data['api'].values)\n",
    "    with open(\"API-Tfidf_%s.pkl\"%str(n_range),'wb') as f:\n",
    "        pickle.dump(api_tfidf,f)\n",
    "        \n",
    "else:\n",
    "    with open(\"API-Tfidf_%s.pkl\"%str(n_range),'rb')  as f:\n",
    "        api_tfidf=pickle.load(f)\n",
    "\n",
    "train_x=api_tfidf.transform(train_data['api'].values)\n",
    "#test_x=api_tfidf.transform(test_data['api'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.shape,train_Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train_x\n",
    "train_Y = df['label'].values\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "import lightgbm as lgb\n",
    "\n",
    "skf = StratifiedKFold(train_Y, n_folds=5, shuffle=True, random_state=2018)\n",
    "for i,(tr_idx,val_idx) in enumerate(skf):\n",
    "        print('FOLD: ',i)\n",
    "        X_train,X_train_label = train_X[tr_idx],train_Y[tr_idx]\n",
    "        X_val,X_val_label = train_X[val_idx],train_Y[val_idx]\n",
    "        dtrain = lgb.Dataset(X_train,X_train_label) \n",
    "        dval   = lgb.Dataset(X_val,X_val_label, reference = dtrain)   \n",
    "        params = {\n",
    "                'task':'train', \n",
    "                'boosting_type':'gbdt',\n",
    "                'num_leaves': 15,\n",
    "                'objective': 'multiclass',\n",
    "                'num_class':8,\n",
    "                'learning_rate': 0.01,\n",
    "                'feature_fraction': 0.85,\n",
    "                'subsample':0.85,\n",
    "                'num_threads': 28,\n",
    "                'metric':'multi_logloss',\n",
    "                'seed':2018\n",
    "            }\n",
    "        model = lgb.train(params, \n",
    "                          dtrain, \n",
    "                          num_boost_round=100000,\n",
    "                          valid_sets=[dval],\n",
    "                          verbose_eval=100, \n",
    "                          early_stopping_rounds=100)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
